/**
 * 文件名:DynastyCrawler.java
 * 包名:com.wx.demo.wcd.crawler
 * 创建时间:2017年4月10日上午9:56:22
 * <p>Copyright ®医界互联 TUS 平台版权所有。</p>
 *
 */

package com.wx.demo.wcd.crawler;

import org.apache.log4j.Logger;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.context.support.AbstractApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;

import com.wx.demo.wcd.service.AuthorService;
import com.wx.demo.wcd.service.impl.AuthorServiceImpl;

/**
 * 类名:DynastyCrawler <br/>
 * 〈功能简述〉朝代爬取. <br/>
 * 〈功能详细描述〉.<br/>
 * 创建时间: 2017年4月10日 上午9:56:22 <br/>
 * 
 * @author 王笑 Email:wangxiao@yijiehulian.com.
 * @version
 * @since JDK 1.8
 * @see
 */
public class DynastyCrawler extends BreadthCrawler {

	private Logger logger = Logger.getLogger(getClass());

	/**
	 * 读取Spring配置信息
	 */
	AbstractApplicationContext ctx = new ClassPathXmlApplicationContext(
			new String[] { "classpath*:/spring/spring-dao.xml",
					"classpath*:/spring/spring-mvc.xml",
					"classpath*:/spring/spring-service.xml" });
	AuthorService authorService = (AuthorServiceImpl) ctx
			.getBean("authorService");

	public static void main(String[] args) throws Exception {
		DynastyCrawler crawler = new DynastyCrawler("depth_crawler", true);
		// 需要爬取的URL
		crawler.addSeed(new CrawlDatum(
				"http://so.gushiwen.org/shiwen/tags.aspx").meta("depth", "1"));
		/* 线程数 */
		crawler.setThreads(50);
		/* 设置每次迭代中爬取数量的上限 */
		crawler.setTopN(10);
		// 爬取深度
		crawler.start(5);
	}

	public DynastyCrawler(String crawlPath, boolean autoParse) {
		super(crawlPath, autoParse);
	}

	@Override
	public void visit(Page page, CrawlDatums next) {

		// 有效节点信息
		Elements ems = page.select("div[class=main3]")
				.select("div[class=typeleft]").select("div[id=tags]")
				.select("a");
		for (Element et : ems) {
			String tagName = et.text().trim();
			logger.info("标签名称:\n" + tagName);
		}

	}

}
